import pandas as pd
import requests
from lxml import etree

# 构造url地址
url = 'https://movie.douban.com/top250'

# 请求头
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36 Edg/93.0.961.38',
    'Cookie': 'bid=Qv1XtVaJyz4; ap_v=0,6.0'
}
# 获取响应并保存目标地址为html文件
response = requests.get(url, headers=headers)
response.encoding = 'utf-8'
with open('topmovie.html', 'w', encoding='utf-8') as f:
    f.write(response.text)

with open('topmovie.html', 'r', encoding='utf-8') as f:
    html = f.read()
# 对获取到的html文档使用 etree.HTML 进行解析，使其变成 Element对象
html = etree.HTML(html)
movie_list = html.xpath('//ol[@class="grid_view"]/li')

# 初始化
author_list = []
form_list = []
star_list = []

# xpath提取元素
for movie in movie_list:

    # 标题
    title = movie.xpath('//span[@class="title"][1]/text()')

    # 主创
    author = movie.xpath('//p[@class=""]/text()[1]')
    form = movie.xpath('//p[@class=""]/text()[2]')
    # 星级
    star_str = movie.xpath('//div[@class="star"]/span[1]/@class')


#  数据预处理
for i in author:
    author_list.append(i.strip())
for i in form:
    # form = i.split('/')
    form_list.append(i.split('/')[-1])
# 星级需要根据一位数或者二位数做出转换
for rating in star_str:
    rating = rating.split('-')[0]

    # 星级
    if len(rating[6:]) >= 2:
        star = int(rating[6:]) / 10  # rating.45的情况
    else:
        star = int(rating[6:])  # rating5的情况
    star_list.append(star)


# 输出到excel
dict = {'电影名': title, '主创': author_list,'类型':form_list,'星级':star}
# dict = {'电影名': title, '主创': author_list,'类型':type,'星级':star}
results1 = pd.DataFrame(dict, index=[i+1 for i in range(len(title))])
results1.to_excel('results.xlsx')

